package crawler.web.utils;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import library.Tracer;

import org.apache.commons.httpclient.Cookie;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.lang.ArrayUtils;

import crawler.web.beans.CrawlURL;
import crawler.web.component.HtmlDigger;
import crawler.web.engine.config.ConfigBatchInit;

public class MyHttpClient {
	private static List<String> userAgentList = ConfigBatchInit.getStringList("userAgentList");
	static {
		Collections.shuffle(userAgentList);
		userAgentList = Collections.synchronizedList(userAgentList);
	}

	private HttpClient hc = new HttpClient();
	private Pattern compile = Pattern.compile("<meta.*?charset=(\")?(.*?)\".*?>");
	private String userAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.14) Gecko/20080404 Firefox/2.0.0.14";

	public MyHttpClient() {
		hc.getHttpConnectionManager().getParams().setConnectionTimeout(1000 * 60);
		hc.getHttpConnectionManager().getParams().setSoTimeout(1000 * 60);
		userAgent = userAgentList.remove(0);
		userAgentList.add(userAgent);
		Tracer.info(">>新建http client: >>" + userAgent);
		hc.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
		hc.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
		hc.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 30000);
	}

	public MyHttpClient(String ip, int port) {
		this();
		hc.getHostConfiguration().setProxy(ip, port);
	}

	public String getHtml(CrawlURL url) throws HttpException, IOException {
		return getHtml(url, 0);
	}

	private String getHtml(CrawlURL url, int redirCount) throws HttpException, IOException {
		if (redirCount >= 5)
			return null;

		String page = null;
		GetMethod httpGet = null;
		try {
			httpGet = new GetMethod(url.getOriUrl());
			httpGet.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
			httpGet.setRequestHeader("Accept-Language", "zh-cn");
			httpGet.setRequestHeader("Cache-Control", "max-age=0");
			httpGet.setRequestHeader("Connection", " Keep-Alive");
			httpGet.setRequestHeader("Host", HtmlDigger.getSiteDomainURL(url.getOriUrl()));
			httpGet.setRequestHeader("Referer", url.getOriUrl());
			httpGet.setRequestHeader("User-Agent", userAgent);

			Tracer.debug("发起http请求");
			Cookie[] cookies = hc.getState().getCookies();
			if (ArrayUtils.isNotEmpty(cookies)) {
				hc.getState().addCookies(cookies);
			}
			Tracer.trace("cookie", "Cookies: " + Arrays.toString(cookies));
			int statusCode = hc.executeMethod(httpGet);
			Tracer.debug("http请求结束");

			Tracer.info("请求  " + url.getOriUrl() + "  状态码为: " + statusCode);
			// 针对状态码进行处理 (简单起见，只处理返回值为200的状态码)
			if (statusCode == HttpStatus.SC_OK) {
				InputStream is = new BufferedInputStream(httpGet.getResponseBodyAsStream());

				List<Byte> list = new ArrayList<Byte>(147216);
				int read;
				while ((read = is.read()) != -1) {
					list.add((byte) read);
				}

				byte[] bs = new byte[list.size()];
				for (int i = 0; i < list.size(); i++)
					bs[i] = list.get(i).byteValue();

				String charset = "utf-8";
				page = new String(bs, charset);
				Matcher matcher = compile.matcher(page);
				while (matcher.find()) {
					charset = matcher.group(2);
				}
				if (!charset.equals("utf-8"))
					page = new String(bs, charset);
				url.setEncoding(charset);
			}
			//若需要转向，则进行转向操作
			else if ((statusCode == HttpStatus.SC_MOVED_TEMPORARILY) || //
			(statusCode == HttpStatus.SC_MOVED_PERMANENTLY) || //
			(statusCode == HttpStatus.SC_SEE_OTHER) || //
			(statusCode == HttpStatus.SC_TEMPORARY_REDIRECT)) {
				//读取新的URL地址
				Header header = httpGet.getResponseHeader("location");
				if (header != null) {
					String newUrl = header.getValue();
					url.setOriUrl(newUrl);
					page = getHtml(url, redirCount++);
				}
			} else if (statusCode == HttpStatus.SC_NOT_FOUND) {
				page = "404";
			}
		} finally {
			if (httpGet != null) {
				httpGet.releaseConnection();
			}
		}
		return page;
	}

	protected void retainCookies(String[] cookieNames) {
		Cookie[] cookies = hc.getState().getCookies();
		ArrayList<Cookie> retainCookies = new ArrayList<Cookie>();
		for (Cookie cookie : cookies) {
			if (Arrays.binarySearch(cookieNames, cookie.getName()) >= 0) {
				retainCookies.add(cookie);
			}
		}
		hc.getState().clearCookies();
		hc.getState().addCookies(retainCookies.toArray(new Cookie[0]));
	}

	protected void removeCookies(String[] cookieNames) {
		Cookie[] cookies = hc.getState().getCookies();
		ArrayList<Cookie> retainCookies = new ArrayList<Cookie>();
		for (Cookie cookie : cookies) {
			if (Arrays.binarySearch(cookieNames, cookie.getName()) < 0) {
				retainCookies.add(cookie);
			}
		}
		hc.getState().clearCookies();
		hc.getState().addCookies(retainCookies.toArray(new Cookie[0]));
	}
}
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//
//